package main

import (
	"fmt"
	"my-crawler/concurrence/engine"
	"my-crawler/concurrence/fetcher"
	"my-crawler/concurrence/task"
	"my-crawler/concurrence/task/gzh_comments_num/dao"
	"my-crawler/concurrence/task/gzh_comments_num/parser"
	"my-crawler/concurrence/task/gzh_comments_num/saver"
)

//concurrence 并行版
func main() {
	//查询返回一个结果
	res, header, reqUrl := dao.GetModels(fmt.Sprintf("%d", task.BeginCommentSqlId))
	//保存 Or 更新数据
	saveChan := saver.Saver()
	//启动Worker等内核
	newEngine := engine.ConcurrenceEngine{
		WorkerCount: 10,
		SaverChan:   saveChan,
		Scheduler:   &engine.ConcurrenceScheduler{},
	}

	//初始化初次请求
	newEngine.Initial(engine.CrawlerRequest{
		ReqData:  engine.CreateReqData(header, res, reqUrl), //数据初始化
		HttpFunc: &fetcher.GetReq{},                         //请求
		Parser:   parser.GetCommentIdParser,
	})
}
